# 导入数据请求模块，第三方模块 需要在cmd里面 pip install requests
import json
import random
import time

import redis
import requests
import urllib.parse

from bs4 import BeautifulSoup

"""
1. 发送请求，对于模拟浏览器对于URL地址发送请求
    - 如何批量替换内容
        (.*?):(.*)
        '$1':'$2',
"""

# 商品列表
SHOP_URL = 'https://www.dianping.com/chongqing/ch85/p'
redisClient = redis.Redis(host='127.0.0.1', port=6379, db=0, decode_responses=True, password=123456)
user_agents = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/57.0.4',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/16.16299',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 '
    'Safari/605.1.15'
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
    'Chrome/127.0.0.0'
    'Safari/537.36',
]


def request_url(url):
    try:
        # 确定请求URL地址
        # url = "https://www.dianping.com/chongqing/ch85"
        # 请求参数，字段数据类型，构建完整键值对
        data = {}
        # 请求头: 伪装，把python代码伪装（模拟）成浏览器发送请求
        headers = {
            # user-agent 用户代理，表示浏览器基本身份标识
            'user-agent': random.choice(user_agents),
            'cookie': 'logan_session_token=9j4gzvna00i8opvzutht; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; '
                      '_lxsdk_cuid=191459676fbc8-0bf09f35f59bb5-26001e51-1fa400-191459676fbc8; '
                      '_lxsdk=191459676fbc8-0bf09f35f59bb5-26001e51-1fa400-191459676fbc8; '
                      '_hc.v=26a335d2-1c24-d20b-7bde-67789d8b317b.1723449375; fspop=test; s_ViewType=10; '
                      'WEBDFPID=5uy60v06266y5400z47uu64u28wv8z6w8085403y02y9795858y7v8v0-2038809389651'
                      '-1723449389075AOKAQKMfd79fef3d01d5e9aadc18ccd4d0c95074117; '
                      'qruuid=c1c73b9e-4386-4366-803b-064852411375; '
                      'dper=0202e7216dd6f378dfcefadaa41e92985ff3b8f4ec79e8ce7f5f709cb77b355e395791988f20f8adac3940d24e7905464ce152d7bddfd225297e00000000042200003597d72b26fa0fa14d8ae432694132836cfe2e45b3d328835b3228b16062fa5cf470a3be5a39976fd1ae38caa619d698; ll=7fd06e815b796be3df069dec7836c3df; Hm_lvt_602b80cf8079ae6591966cc70a3940e7=1723449421; HMACCOUNT=74CADAA2AD777BAD; cy=9; cye=chongqing; Hm_lpvt_602b80cf8079ae6591966cc70a3940e7=1723450071; _lxsdk_s=191459676fb-99a-519-4a5%7C%7C204',
        }
        proxy = '127.0.0.1:7890'
        proxies = {
            'https': 'http://' + proxy,  # 查找到vpn在本机使用的http代理端口
            'http': 'http://' + proxy,  # 查找到你的vpn在本机使用的https代理端口
        }
        # 发送请求 <Response [403]> 返回响应对象 403状态码 表示请求你没有访问权限《没有请求成功》
        response = requests.get(url=url, headers=headers, params=data, proxies=proxies, timeout=10)
        if response.status_code == 200:
            document = BeautifulSoup(response.content, 'html.parser')
            return document
    except Exception as e:
        print(f"An error occurred: {e}")
        raise e


def get_shop_list():
    page = redisClient.get('shop:page')
    if page is None:
        page = 1
    for index in range(int(page), 51):
        url = SHOP_URL + str(index)
        print(url)
        document = request_url(url)
        shopList = document.find("div", {"class": "shop-list J_shop-list shop-all-list"}).find_next("ul").find_all("li")
        for shop in shopList:
            shopUrl = shop.find_next("div", {"class": "pic"}).find_next("a").attrs.get("href")
            shopId = shopUrl.split('/')[-1]
            shopHome = request_url(shopUrl)
            shop_page_handler(shopHome, shopId)
            time.sleep(random.randrange(1, 5))
        redisClient.set('shop:page', index)
        time.sleep(random.randrange(1, 4))


def shop_page_handler(document, shopId):
    shopName = document.find("h1", {"class": "shop-name"}).contents[0]
    address = document.find("div", {"id": "J_map-show"}).find_next("span", {"class": "item"}).get_text()
    telephone = document.find("p", {"class": "expand-info tel"}).contents[2]
    print(shopId, shopName, telephone, address)
    data = json.dumps([
        str.replace(shopName, ' ', ''),
        str.replace(address, ' ', ''),
        str.replace(telephone, ' ', '')
    ])
    if redisClient.sismember('shop:id', shopId) == 0:
        redisClient.sadd("shop:id", shopId)
        redisClient.rpush("shop:list", data)


if __name__ == '__main__':
    get_shop_list()
